import os
import re
import string
from collections import Counter
import matplotlib.pyplot as plt
import nltk.data
import plotly.graph_objs as go
import plotly.offline as py
import pandas as pd
import numpy as np
import seaborn as sns
sns.set(style='white')
import warnings
warnings.filterwarnings("ignore")
import logging
logging.getLogger('lda').setLevel(logging.WARNING)
os.environ["KMP_OUPLICATE_OK"]="TRUE"
pd.set_option('display.max_columns',1000)
pd.set_option('display.width',1000)
pd.set_option('display.max_colwidth',1000)

train=pd.read_csv("data/train.tsv",sep="\t")
test=pd.read_csv("data/test.tsv",sep="\t")
print(train.shape)
print(test.shape)
print(train.dtypes)
print(train.head())
print(train.price.describe())
plt.subplot(1,2,1)
(train['price']).plot.hist(bins=50,figsize=(20,10),edgecolor='white',range=[0,2500])
plt.xlabel('price+',fontsize=17)
plt.ylabel('frequency',fontsize=17)
plt.tick_params(labelsize=15)
plt.title('Price Distribution-Training set',fontsize=17)

plt.subplot(1,2,2)
np.log(train['price']+1).plot.hist(bins=50,figsize=(20,10),edgecolor='white',range=[0,2500])
plt.xlabel('log(price+1)',fontsize=17)
plt.ylabel('frequency',fontsize=17)
plt.tick_params(labelsize=15)
plt.title('log(Price) Distribution-Training set',fontsize=17)
plt.show()

print(train.shipping.value_counts()/len(train))
prc_shippingBySeller=train.loc[train.shipping==0,'price']
prc_shippingByBuyer=train.loc[train.shipping==1,'price']
fig,ax=plt.subplots(figsize=(20,10))
ax.hist(np.log(prc_shippingBySeller+1),color='#8CB4E1',alpha=1.0,bins=50,label='Seller pay shipping')
ax.hist(np.log(prc_shippingByBuyer+1),color='#007D00',alpha=1.0,bins=50,label='Buyer pay shipping')
ax.set(title='Histogram Comparison',ylabel='% of Dateset in Bin')
plt.legend()
plt.xlabel('log(price+1',fontsize=17)
plt.xlabel('frequency',fontsize=17)
plt.title("Price Distribution by Shipping Type",fontsize=17)
plt.tick_params(labelsize=15)
plt.show()


#商品类别太多合并下
def split_cat(text):
    try:
        return text.split("/")
    except:
        return "No Label", "No Label", "No Label"

train['general_cat'], train['subcat_1'], train['subcat_2'] = zip(*train['category_name'].apply(lambda x: split_cat(x)))
print(train.head())
test['general_cat'], test['subcat_1'], test['subcat_2'] = zip(*test['category_name'].apply(lambda x: split_cat(x)))
#print('There are %d unique general_cat' % train['general_cat'].nunique())
#print('There are %d unique first sub-categories' % train['subcat_1'].nunique())
#print('There are %d unique second sub-categories' % train['subcat_2'].nunique())
#主类别分布情况
x = train['general_cat'].value_counts().index.astype('str')
y = train['general_cat'].value_counts().values
pct = [('%.2f' % (v * 100)) + '%' for v in (y / len(train))]
tracel = go.Bar(x=x, y=y, test=pct)
layout = dict(title="Number of Items by Main Category", yaxis=dict(title='Count'), xaxis=dict(title='Category'))
fig = dict(data=[tracel], layout=layout)
py.plot(fig)
#前15个子类别分布情况
x = train['general_cat'].value_counts().index.astype('str')[:15]
y = train['general_cat'].value_counts().values[:15]
pct = [('%.2f' % (v * 100)) + '%' for v in (y / len(train))][:15]
tracel = go.Bar(x=x, y=y, test=pct, marker=dict(color=y, colorscale='Portland', showscale=True, reversescale=False))
layout = dict(title="Number of Items by Sub Category(Top 15)", yaxis=dict(title='Count'), xaxis=dict(title='SubCategory'))
fig = dict(data=[tracel], layout=layout)
py.plot(fig)